{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\nDeploy Pretrained Vision Model from MxNet on VTA\n================================================\n**Author**: `Thierry Moreau <https://homes.cs.washington.edu/~moreau/>`_\n\nThis tutorial provides an end-to-end demo, on how to run ImageNet classification\ninference onto the VTA accelerator design to perform ImageNet classification tasks.\nIt showcases Relay as a front end compiler that can perform quantization (VTA\nonly supports int8/32 inference) as well as graph packing (in order to enable\ntensorization in the core) to massage the compute graph for the hardware target.\n\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Install dependencies\n--------------------\nTo use the autotvm package in tvm, we need to install some extra dependencies.\n(change \"3\" to \"2\" if you use python2):\n\n.. code-block:: bash\n\n  pip3 install --user mxnet requests \"Pillow<7\"\n\nNow return to the python code. Import packages.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from __future__ import absolute_import, print_function\n\nimport argparse, json, os, requests, sys, time\nfrom io import BytesIO\nfrom os.path import join, isfile\nfrom PIL import Image\n\nfrom mxnet.gluon.model_zoo import vision\nimport numpy as np\nfrom matplotlib import pyplot as plt\n\nimport tvm\nfrom tvm import te\nfrom tvm import rpc, autotvm, relay\nfrom tvm.contrib import graph_executor, utils, download\nfrom tvm.contrib.debugger import debug_executor\nfrom tvm.relay import transform\n\nimport vta\nfrom vta.testing import simulator\nfrom vta.top import graph_pack\n\n\n# Make sure that TVM was compiled with RPC=1\nassert tvm.runtime.enabled(\"rpc\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Define the platform and model targets\n-------------------------------------\nExecute on CPU vs. VTA, and define the model.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Load VTA parameters from the 3rdparty/vta-hw/config/vta_config.json file\nenv = vta.get_env()\n\n# Set ``device=arm_cpu`` to run inference on the CPU\n# or ``device=vta`` to run inference on the FPGA.\ndevice = \"vta\"\ntarget = env.target if device == \"vta\" else env.target_vta_cpu\n\n# Dictionary lookup for when to start/end bit packing\npack_dict = {\n    \"resnet18_v1\": [\"nn.max_pool2d\", \"nn.global_avg_pool2d\"],\n    \"resnet34_v1\": [\"nn.max_pool2d\", \"nn.global_avg_pool2d\"],\n    \"resnet18_v2\": [\"nn.max_pool2d\", \"nn.global_avg_pool2d\"],\n    \"resnet34_v2\": [\"nn.max_pool2d\", \"nn.global_avg_pool2d\"],\n    \"resnet50_v2\": [\"nn.max_pool2d\", \"nn.global_avg_pool2d\"],\n    \"resnet101_v2\": [\"nn.max_pool2d\", \"nn.global_avg_pool2d\"],\n}\n\n# Name of Gluon model to compile\n# The ``start_pack`` and ``stop_pack`` labels indicate where\n# to start and end the graph packing relay pass: in other words\n# where to start and finish offloading to VTA.\nmodel = \"resnet18_v1\"\nassert model in pack_dict"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Obtain an execution remote\n--------------------------\nWhen target is 'pynq', reconfigure FPGA and runtime.\nOtherwise, if target is 'sim', execute locally.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "if env.TARGET not in [\"sim\", \"tsim\", \"intelfocl\"]:\n\n    # Get remote from tracker node if environment variable is set.\n    # To set up the tracker, you'll need to follow the \"Auto-tuning\n    # a convolutional network for VTA\" tutorial.\n    tracker_host = os.environ.get(\"TVM_TRACKER_HOST\", None)\n    tracker_port = os.environ.get(\"TVM_TRACKER_PORT\", None)\n    # Otherwise if you have a device you want to program directly from\n    # the host, make sure you've set the variables below to the IP of\n    # your board.\n    device_host = os.environ.get(\"VTA_RPC_HOST\", \"192.168.2.99\")\n    device_port = os.environ.get(\"VTA_RPC_PORT\", \"9091\")\n    if not tracker_host or not tracker_port:\n        remote = rpc.connect(device_host, int(device_port))\n    else:\n        remote = autotvm.measure.request_remote(\n            env.TARGET, tracker_host, int(tracker_port), timeout=10000\n        )\n\n    # Reconfigure the JIT runtime and FPGA.\n    # You can program the FPGA with your own custom bitstream\n    # by passing the path to the bitstream file instead of None.\n    reconfig_start = time.time()\n    vta.reconfig_runtime(remote)\n    vta.program_fpga(remote, bitstream=None)\n    reconfig_time = time.time() - reconfig_start\n    print(\"Reconfigured FPGA and RPC runtime in {0:.2f}s!\".format(reconfig_time))\n\n# In simulation mode, host the RPC server locally.\nelse:\n    remote = rpc.LocalSession()\n\n    if env.TARGET in [\"intelfocl\"]:\n        # program intelfocl aocx\n        vta.program_fpga(remote, bitstream=\"vta.bitstream\")\n\n# Get execution context from remote\nctx = remote.ext_dev(0) if device == \"vta\" else remote.cpu(0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Build the inference graph executor\n----------------------------------\nGrab vision model from Gluon model zoo and compile with Relay.\nThe compilation steps are:\n\n1. Front end translation from MxNet into Relay module.\n2. Apply 8-bit quantization: here we skip the first conv layer,\n   and dense layer which will both be executed in fp32 on the CPU.\n3. Perform graph packing to alter the data layout for tensorization.\n4. Perform constant folding to reduce number of operators (e.g. eliminate batch norm multiply).\n5. Perform relay build to object file.\n6. Load the object file onto remote (FPGA device).\n7. Generate graph executor, `m`.\n\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Load pre-configured AutoTVM schedules\nwith autotvm.tophub.context(target):\n\n    # Populate the shape and data type dictionary for ImageNet classifier input\n    dtype_dict = {\"data\": \"float32\"}\n    shape_dict = {\"data\": (env.BATCH, 3, 224, 224)}\n\n    # Get off the shelf gluon model, and convert to relay\n    gluon_model = vision.get_model(model, pretrained=True)\n\n    # Measure build start time\n    build_start = time.time()\n\n    # Start front end compilation\n    mod, params = relay.frontend.from_mxnet(gluon_model, shape_dict)\n\n    # Update shape and type dictionary\n    shape_dict.update({k: v.shape for k, v in params.items()})\n    dtype_dict.update({k: str(v.dtype) for k, v in params.items()})\n\n    if target.device_name == \"vta\":\n        # Perform quantization in Relay\n        # Note: We set opt_level to 3 in order to fold batch norm\n        with tvm.transform.PassContext(opt_level=3):\n            with relay.quantize.qconfig(global_scale=8.0, skip_conv_layers=[0]):\n                mod = relay.quantize.quantize(mod, params=params)\n            # Perform graph packing and constant folding for VTA target\n            assert env.BLOCK_IN == env.BLOCK_OUT\n            # do device annotation if target is intelfocl or sim\n            relay_prog = graph_pack(\n                mod[\"main\"],\n                env.BATCH,\n                env.BLOCK_OUT,\n                env.WGT_WIDTH,\n                start_name=pack_dict[model][0],\n                stop_name=pack_dict[model][1],\n                device_annot=(env.TARGET == \"intelfocl\"),\n            )\n    else:\n        relay_prog = mod[\"main\"]\n\n    # Compile Relay program with AlterOpLayout disabled\n    if target.device_name != \"vta\":\n        with tvm.transform.PassContext(opt_level=3, disabled_pass={\"AlterOpLayout\"}):\n            graph, lib, params = relay.build(\n                relay_prog, target=target, params=params, target_host=env.target_host\n            )\n    else:\n        if env.TARGET == \"intelfocl\":\n            # multiple targets to run both on cpu and vta\n            target = {\"cpu\": env.target_vta_cpu, \"ext_dev\": target}\n        with vta.build_config(opt_level=3, disabled_pass={\"AlterOpLayout\"}):\n            graph, lib, params = relay.build(\n                relay_prog, target=target, params=params, target_host=env.target_host\n            )\n\n    # Measure Relay build time\n    build_time = time.time() - build_start\n    print(model + \" inference graph built in {0:.2f}s!\".format(build_time))\n\n    # Send the inference library over to the remote RPC server\n    temp = utils.tempdir()\n    lib.export_library(temp.relpath(\"graphlib.tar\"))\n    remote.upload(temp.relpath(\"graphlib.tar\"))\n    lib = remote.load_module(\"graphlib.tar\")\n\n    if env.TARGET == \"intelfocl\":\n        ctxes = [remote.ext_dev(0), remote.cpu(0)]\n        m = graph_executor.create(graph, lib, ctxes)\n    else:\n        # Graph runtime\n        m = graph_executor.create(graph, lib, ctx)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Perform image classification inference\n--------------------------------------\nWe run classification on an image sample from ImageNet\nWe just need to download the categories files, `synset.txt`\nand an input test image.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Download ImageNet categories\ncateg_url = \"https://github.com/uwsampl/web-data/raw/main/vta/models/\"\ncateg_fn = \"synset.txt\"\ndownload.download(join(categ_url, categ_fn), categ_fn)\nsynset = eval(open(categ_fn).read())\n\n# Download test image\nimage_url = \"https://homes.cs.washington.edu/~moreau/media/vta/cat.jpg\"\nimage_fn = \"cat.png\"\ndownload.download(image_url, image_fn)\n\n# Prepare test image for inference\nimage = Image.open(image_fn).resize((224, 224))\nplt.imshow(image)\nplt.show()\nimage = np.array(image) - np.array([123.0, 117.0, 104.0])\nimage /= np.array([58.395, 57.12, 57.375])\nimage = image.transpose((2, 0, 1))\nimage = image[np.newaxis, :]\nimage = np.repeat(image, env.BATCH, axis=0)\n\n# Set the network parameters and inputs\nm.set_input(**params)\nm.set_input(\"data\", image)\n\n# Perform inference and gather execution statistics\n# More on: :py:method:`tvm.runtime.Module.time_evaluator`\nnum = 4  # number of times we run module for a single measurement\nrep = 3  # number of measurements (we derive std dev from this)\ntimer = m.module.time_evaluator(\"run\", ctx, number=num, repeat=rep)\n\nif env.TARGET in [\"sim\", \"tsim\"]:\n    simulator.clear_stats()\n    timer()\n    sim_stats = simulator.stats()\n    print(\"\\nExecution statistics:\")\n    for k, v in sim_stats.items():\n        # Since we execute the workload many times, we need to normalize stats\n        # Note that there is always one warm up run\n        # Therefore we divide the overall stats by (num * rep + 1)\n        print(\"\\t{:<16}: {:>16}\".format(k, v // (num * rep + 1)))\nelse:\n    tcost = timer()\n    std = np.std(tcost.results) * 1000\n    mean = tcost.mean * 1000\n    print(\"\\nPerformed inference in %.2fms (std = %.2f) for %d samples\" % (mean, std, env.BATCH))\n    print(\"Average per sample inference time: %.2fms\" % (mean / env.BATCH))\n\n# Get classification results\ntvm_output = m.get_output(0, tvm.nd.empty((env.BATCH, 1000), \"float32\", remote.cpu(0)))\nfor b in range(env.BATCH):\n    top_categories = np.argsort(tvm_output.numpy()[b])\n    # Report top-5 classification results\n    print(\"\\n{} prediction for sample {}\".format(model, b))\n    print(\"\\t#1:\", synset[top_categories[-1]])\n    print(\"\\t#2:\", synset[top_categories[-2]])\n    print(\"\\t#3:\", synset[top_categories[-3]])\n    print(\"\\t#4:\", synset[top_categories[-4]])\n    print(\"\\t#5:\", synset[top_categories[-5]])\n    # This just checks that one of the 5 top categories\n    # is one variety of cat; this is by no means an accurate\n    # assessment of how quantization affects classification\n    # accuracy but is meant to catch changes to the\n    # quantization pass that would accuracy in the CI.\n    cat_detected = False\n    for k in top_categories[-5:]:\n        if \"cat\" in synset[k]:\n            cat_detected = True\n    assert cat_detected"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}